import hashlib
import json

import pymongo
import redis
import requests
from lxml import etree


class WeiBo:
    def __init__(self):
        self.redis_conn = redis.Redis(host='127.0.0.1', port=6379, db=1)
        self.mongo_client = pymongo.MongoClient(host='127.0.0.1', port=27017)
        self.collection = self.mongo_client['py_spider']['weibo']
        self.redis_set_key = 'weibo_unique_set'
        self.headers = {
            "authority": "s.weibo.com",
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
            "cache-control": "no-cache",
            "pragma": "no-cache",
            "referer": "https://www.baidu.com/link?url=4GteyNfbEATPxTQ8w68fAONsOuEVK7g6bsncdk1Fp37IGPgPjH8HHjz3TvRBp0PL-7RwGOPXGUir_G2SSnA4e2OFNsRB7CwSQixuqUuTSJA2JeabZ1p2GCcYeNFxN6pA&wd=&eqid=d1d0756c0082ce3600000006660bc1ca",
            "sec-fetch-dest": "document",
            "sec-fetch-mode": "navigate",
            "sec-fetch-site": "cross-site",
            "sec-fetch-user": "?1",
            "upgrade-insecure-requests": "1",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0"
        }
        self.cookies = {
            "UOR": "www.baidu.com,weibo.com,www.baidu.com",
            "SINAGLOBAL": "5194396662223.659.1695717088850",
            "SUB": "_2AkMSGMf0f8NxqwFRmfkWyW3maoVwwwDEieKkRDYvJRMxHRl-yj9kqn1YtRB6OZjpGyXUbZUsG6WPbYjorGyd4bGNlMTu",
            "SUBP": "0033WrSXqPxfM72-Ws9jqgMF55529P9D9W54o0uTWkOyYQlz.7i0FrgS",
            "_s_tentry": "www.weibo.com",
            "Apache": "5460029126267.418.1712046321540",
            "ULV": "1712046321563:4:1:1:5460029126267.418.1712046321540:1707186992190"
        }
        self.url = "https://s.weibo.com/top/summary/&wd=&eqid=cf426f1700557c5c0000000365acf8df"

    def get_data(self):
        response = requests.get(self.url, headers=self.headers, cookies=self.cookies)

        response.encoding = response.apparent_encoding
        # print(response.text)
        print(response)

        html = etree.HTML(response.text)

        content_list = html.xpath("//div[@id='pl_top_realtimehot']//tbody/tr[position()>1]")
        domain = "https://s.weibo.com"
        for content in content_list:
            item = dict()
            item['title'] = content.xpath(".//td[@class='td-02']/a/text()")[0].strip()
            item['url'] = domain + content.xpath(".//td[@class='td-02']/a/@href")[0].strip()
            item['hot'] = content.xpath(".//td[@class='td-02']/span/text()")[0].strip()
            md5 = hashlib.md5(json.dumps(item, ensure_ascii=False).encode()).hexdigest()
            # redis去重，不存在才入库
            if self.redis_conn.sadd(self.redis_set_key, md5) != 0:
                self.collection.insert_one(item)
                print("入库成功：", item)

    def __del__(self):
        self.mongo_client.close()
        self.redis_conn.close()


if __name__ == '__main__':
    w = WeiBo()
    w.get_data()
